package com.coderfans.pageprocessor.pageprocessor;

import com.coderfans.datagithub.dao.UserDao;
import org.springframework.context.ApplicationContext;
import org.springframework.context.support.ClassPathXmlApplicationContext;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;

/**
 * Created by ${fanchunshuai} on 2016-7-27.
 *
 * @version: V1.0
 * @Desc:
 * @Copyright (c) 2016 58到家-版权所有
 */
public class GithubRepoPageProcessor implements PageProcessor {


    private static UserDao userDao;

    //步骤一：抓取网站的相关配置 ： 包括编码 抓取间隔 重试次数等
    private Site site = Site.me().setRetryTimes(3).setSleepTime(1000);


    //定制爬虫逻辑的核心接口，这里编写抽取逻辑
    public void process(Page page) {
        page.addTargetRequests(page.getHtml().links().regex("(https://github\\.com/\\w+/\\w+)").all());
        page.addTargetRequests(page.getHtml().links().regex("(https://github\\.com/\\w+)").all());
        System.out.println("url = "+page.getUrl().get());
        //保存结果author，这个结果会最终保存到ResultItems中
        /*page.putField("author", page.getUrl().regex("https://github\\.com/(\\w+)/.*").toString());
        page.putField("name", page.getHtml().xpath("//h1[@class='entry-title public']/strong/a/text()").toString());
        String fullName = page.getHtml().xpath("//h1[@class='vcard-names my-3']/div[@class='vcard-fullname']/text()").toString();
        String username = page.getHtml().xpath("//h1[@class='vcard-names my-3']/div[@class='vcard-username']/text()").toString();
        String email = page.getHtml().xpath("//ul[@class='vcard-details']/li[@class='vcard-detail']/a/text()").toString();
        String blogOrwebSite = page.getHtml().xpath("//ul[@class='vcard-details']/li[@aria-label='Blog or website']/a/text()").toString();
        String joinDate = page.getHtml().xpath("//ul[@class='vcard-details']/li[@aria-label='Member since']/local-time[@class='join-date']/text()").toString();
        String userintroduce = page.getHtml().xpath("//div[@class='column one-fourth']//div[@class='user-profile-bio']/div/text()").toString();
        String address  = page.getHtml().xpath("//ul[@class='vcard-details']//li[@aria-label='Home location']/@title").toString();
        String organization  = page.getHtml().xpath("//ul[@class='vcard-details']//li[@aria-label='Organization']/div/text()").toString();


        //github所属组织，
        List<String> organization1s = page.getHtml().xpath("//div[@class='column one-fourth']//div[@class='clearfix']//a/@href").all();

        List<String>  followers = page.getHtml().xpath("//div[@class='column one-fourth']//div[@class='vcard-stats']/a/strong/text()").all();

        System.out.println("followers:"+followers.toString()+",address:"+address+",userintroduce:"+
                userintroduce+",organization:"+organization+",joinDate:"+joinDate);*/



       /* System.out.println("username:"+username+", fullname:"+fullName+",email:"+email+",\nblogOrwebSite:"+blogOrwebSite+",joinDate:"+joinDate
        +",organizations:"+organizations+",organization1s = "+organization1s.toString());*/

        //System.out.println(page.getHtml().getDocument().body());
        /*if (page.getResultItems().get("name")==null){
            //设置skip之后，这个页面的结果不会被Pipeline处理
            page.setSkip(true);
        }*/
        page.putField("readme", page.getHtml().xpath("//div[@id='readme']/tidyText()"));
    }

    public Site getSite() {
        return site;
    }

    public static void main(String[] args) {
        //手动加载spring配置文件
        ApplicationContext applicationContext = new ClassPathXmlApplicationContext("classpath:spring-mybatis.xml");
       /* userDao = applicationContext.getBean(UserDao.class);
        List<UserModel> gitHubUserList = userDao.getUser();
        System.out.println(gitHubUserList.size());*/
        Spider.create(new GithubRepoPageProcessor()).addPipeline(new com.coderfans.datagithub.ConsolePipeline())
                .addUrl("https://github.com/lw-lin")
                .thread(8).run();

    }
}
